{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import tempfile\n",
    "import urllib\n",
    "import pandas as pd\n",
    "import os\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "######################################\n",
    "######### Necessary flags ############\n",
    "######################################\n",
    "\n",
    "max_num_checkpoint = 10\n",
    "num_classes = 2\n",
    "batch_size = 512\n",
    "num_epochs = 10\n",
    "\n",
    "##########################################\n",
    "######## Learning rate flags #############\n",
    "##########################################\n",
    "\n",
    "initial_learning_rate = 0.001\n",
    "learning_rate_decay_factor = 0.95\n",
    "num_epochs_per_decay = 1\n",
    "\n",
    "#########################################\n",
    "########## status flags #################\n",
    "#########################################\n",
    "\n",
    "is_training = False\n",
    "fine_tuning = False\n",
    "online_test = True\n",
    "allow_soft_placement = True\n",
    "log_device_placement = False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "Extracting MNIST_data/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Download and get MNIST dataset(available in tensorflow.contrib.learn.python.learn.datasets.mnist)\n",
    "# It checks and download MNIST if it's not already downloaded then extract it.\n",
    "# The 'reshape' is True by default to extract feature vectors but we set it to false to we get the original images.\n",
    "mnist = input_data.read_data_sets(\"MNIST_data/\", reshape=True, one_hot=False)\n",
    "\n",
    "########################\n",
    "### Data Processing ####\n",
    "########################\n",
    "# Organize the data and feed it to associated dictionaries.\n",
    "data={}\n",
    "\n",
    "data['train/image'] = mnist.train.images\n",
    "data['train/label'] = mnist.train.labels\n",
    "data['test/image'] = mnist.test.images\n",
    "data['test/label'] = mnist.test.labels\n",
    "\n",
    "def extract_samples_Fn(data):\n",
    "    index_list = []\n",
    "    for sample_index in range(data.shape[0]):\n",
    "        label = data[sample_index]\n",
    "        if label == 1 or label == 0:\n",
    "            index_list.append(sample_index)\n",
    "    return index_list\n",
    "\n",
    "\n",
    "# Get only the samples with zero and one label for training.\n",
    "index_list_train = extract_samples_Fn(data['train/label'])\n",
    "\n",
    "\n",
    "# Get only the samples with zero and one label for test set.\n",
    "index_list_test = extract_samples_Fn(data['test/label'])\n",
    "\n",
    "# Reform the train data structure.\n",
    "data['train/image'] = mnist.train.images[index_list_train]\n",
    "data['train/label'] = mnist.train.labels[index_list_train]\n",
    "\n",
    "# Reform the test data structure.\n",
    "data['test/image'] = mnist.test.images[index_list_test]\n",
    "data['test/label'] = mnist.test.labels[index_list_test]\n",
    "\n",
    "# Dimentionality of train\n",
    "dimensionality_train = data['train/image'].shape\n",
    "\n",
    "# Dimensions\n",
    "num_train_samples = dimensionality_train[0]\n",
    "num_features = dimensionality_train[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Training Loss= 0.32686\n",
      "Epoch 2, Training Loss= 0.13760\n",
      "Epoch 3, Training Loss= 0.08637\n",
      "Epoch 4, Training Loss= 0.06380\n",
      "Epoch 5, Training Loss= 0.05090\n",
      "Epoch 6, Training Loss= 0.04240\n",
      "Epoch 7, Training Loss= 0.03636\n",
      "Epoch 8, Training Loss= 0.03186\n",
      "Epoch 9, Training Loss= 0.02838\n",
      "Epoch 10, Training Loss= 0.02562\n",
      "Final Test Accuracy is % 99.95\n"
     ]
    }
   ],
   "source": [
    "#######################################\n",
    "########## Defining Graph ############\n",
    "#######################################\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    ###################################\n",
    "    ########### Parameters ############\n",
    "    ###################################\n",
    "\n",
    "    # global step\n",
    "    global_step = tf.Variable(0, name=\"global_step\", trainable=False)\n",
    "\n",
    "    # learning rate policy\n",
    "    decay_steps = int(num_train_samples / batch_size *\n",
    "                      num_epochs_per_decay)\n",
    "    learning_rate = tf.train.exponential_decay(initial_learning_rate,\n",
    "                                               global_step,\n",
    "                                               decay_steps,\n",
    "                                               learning_rate_decay_factor,\n",
    "                                               staircase=True,\n",
    "                                               name='exponential_decay_learning_rate')\n",
    "    ###############################################\n",
    "    ########### Defining place holders ############\n",
    "    ###############################################\n",
    "    image_place = tf.placeholder(tf.float32, shape=([None, num_features]), name='image')\n",
    "    label_place = tf.placeholder(tf.int32, shape=([None,]), name='gt')\n",
    "    label_one_hot = tf.one_hot(label_place, depth=num_classes, axis=-1)\n",
    "    dropout_param = tf.placeholder(tf.float32)\n",
    "\n",
    "    ##################################################\n",
    "    ########### Model + Loss + Accuracy ##############\n",
    "    ##################################################\n",
    "    # A simple fully connected with two class and a softmax is equivalent to Logistic Regression.\n",
    "    logits = tf.contrib.layers.fully_connected(inputs=image_place, num_outputs = num_classes, scope='fc')\n",
    "\n",
    "    # Define loss\n",
    "    with tf.name_scope('loss'):\n",
    "        loss_tensor = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=label_one_hot))\n",
    "\n",
    "    # Accuracy\n",
    "    # Evaluate the model\n",
    "    prediction_correct = tf.equal(tf.argmax(logits, 1), tf.argmax(label_one_hot, 1))\n",
    "\n",
    "    # Accuracy calculation\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction_correct, tf.float32))\n",
    "\n",
    "    #############################################\n",
    "    ########### training operation ##############\n",
    "    #############################################\n",
    "\n",
    "    # Define optimizer by its default values\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)\n",
    "\n",
    "    # 'train_op' is a operation that is run for gradient update on parameters.\n",
    "    # Each execution of 'train_op' is a training step.\n",
    "    # By passing 'global_step' to the optimizer, each time that the 'train_op' is run, Tensorflow\n",
    "    # update the 'global_step' and increment it by one!\n",
    "\n",
    "    # gradient update.\n",
    "    with tf.name_scope('train_op'):\n",
    "        gradients_and_variables = optimizer.compute_gradients(loss_tensor)\n",
    "        train_op = optimizer.apply_gradients(gradients_and_variables, global_step=global_step)\n",
    "\n",
    "\n",
    "    ############################################\n",
    "    ############ Run the Session ###############\n",
    "    ############################################\n",
    "    session_conf = tf.ConfigProto(\n",
    "        allow_soft_placement=allow_soft_placement,\n",
    "        log_device_placement=log_device_placement)\n",
    "    sess = tf.Session(graph=graph, config=session_conf)\n",
    "\n",
    "    with sess.as_default():\n",
    "\n",
    "        # The saver op.\n",
    "        saver = tf.train.Saver()\n",
    "\n",
    "        # Initialize all variables\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        # The prefix for checkpoint files\n",
    "        checkpoint_prefix = 'model'\n",
    "\n",
    "        # If fie-tuning flag in 'True' the model will be restored.\n",
    "        if fine_tuning:\n",
    "            saver.restore(sess, os.path.join(checkpoint_path, checkpoint_prefix))\n",
    "            print(\"Model restored for fine-tuning...\")\n",
    "\n",
    "        ###################################################################\n",
    "        ########## Run the training and loop over the batches #############\n",
    "        ###################################################################\n",
    "\n",
    "        # go through the batches\n",
    "        test_accuracy = 0\n",
    "        for epoch in range(num_epochs):\n",
    "            total_batch_training = int(data['train/image'].shape[0] / batch_size)\n",
    "\n",
    "            # go through the batches\n",
    "            for batch_num in range(total_batch_training):\n",
    "                #################################################\n",
    "                ########## Get the training batches #############\n",
    "                #################################################\n",
    "\n",
    "                start_idx = batch_num * batch_size\n",
    "                end_idx = (batch_num + 1) * batch_size\n",
    "\n",
    "                # Fit training using batch data\n",
    "                train_batch_data, train_batch_label = data['train/image'][start_idx:end_idx], data['train/label'][\n",
    "                                                                                             start_idx:end_idx]\n",
    "\n",
    "                ########################################\n",
    "                ########## Run the session #############\n",
    "                ########################################\n",
    "\n",
    "                # Run optimization op (backprop) and Calculate batch loss and accuracy\n",
    "                # When the tensor tensors['global_step'] is evaluated, it will be incremented by one.\n",
    "                batch_loss, _, training_step = sess.run(\n",
    "                    [loss_tensor, train_op,\n",
    "                     global_step],\n",
    "                    feed_dict={image_place: train_batch_data,\n",
    "                               label_place: train_batch_label,\n",
    "                               dropout_param: 0.5})\n",
    "\n",
    "                ########################################\n",
    "                ########## Write summaries #############\n",
    "                ########################################\n",
    "\n",
    "\n",
    "                #################################################\n",
    "                ########## Plot the progressive bar #############\n",
    "                #################################################\n",
    "\n",
    "            print(\"Epoch \" + str(epoch + 1) + \", Training Loss= \" + \\\n",
    "                  \"{:.5f}\".format(batch_loss))\n",
    "\n",
    "        ############################################################################\n",
    "        ########## Run the session for pur evaluation on the test data #############\n",
    "        ############################################################################\n",
    "\n",
    "        # Evaluation of the model\n",
    "        test_accuracy = 100 * sess.run(accuracy, feed_dict={\n",
    "            image_place: data['test/image'],\n",
    "            label_place: data['test/label'],\n",
    "            dropout_param: 1.})\n",
    "\n",
    "        print(\"Final Test Accuracy is %% %.2f\" % test_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
